apache2 core dump

--000e0cd5391ca143af047438e2d9
Content-Type: text/plain; charset=ISO-8859-1

Hi all,

I get the following core dumps from the apache:


core 'core.httpd.15844.wenpweb1.500.500.1253648224' of 15844: ./httpd -k
start
----------------- lwp# 5 / thread# 5 --------------------
ff29da3c apr_table_add (6c0348, 453800, 453809, 5, fffc0000, b) + b4
fec521d8 deep_table_copy (6c0348, 0, 5, 4582d4, 38, 1) + 60
fec522f0 store_headers (135c78, 4528b8, 135c80, 4469c8, 135c80, fec522d4) +
1c
fec92860 cache_save_filter (0, 0, 4742f, b8f21380, 4742f, b90155c0) + 538
000454b8 ???????? (44e888, 450838, 7, 0, 0, fea31494)
000454b8 ???????? (45d2a8, 450838, 7, 0, 4488d0, 45083c)
fe8d4d18 ap_proxy_http_process_response (ffffdfec, 4528b8, 3511c8, 456898,
fd0f9a20, 0) + 6c0
fe8d5880 proxy_http_handler (0, 1ba088, 1b9f18, 45d4d8, 0, fd0fbab0) + 290
fe938b4c proxy_run_scheme_handler (4528b8, 1ba088, 1b9f18, 44ebbe, 0, 0) +
64
fe935cf8 proxy_handler (4528b8, 70, 4528b8, 0, 1b9fa0, c9bd8) + 41c
0003b994 ???????? (4528b8, 0, 1b, 1bad40, 1ba7b8, 0)
0003bf38 ???????? (4528b8, 5cc00, 4528b8, fd0fbcdc, fd400028, 1)
00048444 ???????? (4528b8, 77400, 4, 4528b8, 0, 1e8480)
00045b00 ???????? (446980, 446890, 446890, 42, 78088, 1bb820)
00042184 ???????? (446980, 446890, 446890, 42, 446978, 448850)
0004cb3c ???????? (446848, 446890, 1, 2, 448850, 187e48)
0004d28c ???????? (187ef0, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (187ef0, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 6 / thread# 6 --------------------
fef9dbb8 _lwp_makecontext (fcffb418, 1, 1b7740, 3e8, 0, 4ada4) + 14
ff2ac204 apr_wait_for_io_or_timeout (11, fcffb418, 0, 0, fcffb674, 3208) +
54
ff2a7114 apr_socket_sendv (1ee378, fcffb658, 3, fcffb504, 1f9712, b96) + 50
0003a424 ???????? (1ee378, fcffb650, 4, 1f49, fcffb5a8, fcffb6d0)
0003ada8 ???????? (0, 6fd7e8, 78348, 5d2b4, 0, 0)
000454b8 ???????? (1ee9e8, 6fd7e8, 611de, fcffb818, 0, fead0c24)
000454b8 ???????? (1ee9a8, 6fd7e8, 2, 0, 0, 1f0358)
0004b004 ???????? (6fd9e0, 6fd7e8, 0, 0, 780e8, 1ee468)
000454b8 ???????? (6fd9e0, 6fd7e8, 0, 0, 0, 4ada4)
000454b8 ???????? (1f31c8, 6fd7e8, 0, 0, 0, 0)
00030b94 ???????? (1f3198, 6fd7e8, 0, 0, 0, 30abc)
000454b8 ???????? (1f3198, 6fd7e8, 1, 0, 0, 1f06b8)
ff3270f8 apr_brigade_write (6fd7e8, 45648, 1f3198, fcffbaef, 1, 1ee468) +
168
00030f88 ???????? (ffffffff, 2e, 0, 0, 6eddc8, fe8249f8)
fe811f98 status_handler (20c, 0, 0, 1f2380, fe8249fc, fe8249f8) + 710
0003b994 ???????? (1f2380, 0, f, 1f4160, 1f4060, 0)
0003bf38 ???????? (1f2380, 5cc00, 1f2380, fcffbcdc, fd400028, 1)
00048444 ???????? (1f2380, 77400, 4, 1f2380, 78088, 1baf38)
00045b00 ???????? (1ee468, 1ee378, 1ee378, 43, 78088, 1bb820)
00042184 ???????? (1ee468, 1ee378, 1ee378, 43, 1ee460, 1f0338)
0004cb3c ???????? (1ee330, 1ee378, 1, 3, 1f0338, 187e48)
0004d28c ???????? (187f10, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (187f10, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 7 / thread# 7 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee50c00, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bed8d0b0, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fcefbf1c, fcefbf18, 0, 3a48a0, 187e48)
0004d1e4 ???????? (187f30, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (187f30, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 8 / thread# 8 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee50e00, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bed81995, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fcdfbf1c, fcdfbf18, 0, 309068, 187e48)
0004d1e4 ???????? (187f50, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (187f50, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 9 / thread# 9 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee51000, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bed6e868, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fccfbf1c, fccfbf18, 0, 1f0338, 187e48)
0004d1e4 ???????? (187f70, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (187f70, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 10 / thread# 10 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee51200, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bee29721, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fcbfbf1c, fcbfbf18, 0, 3a48a0, 187e48)
0004d1e4 ???????? (187f90, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (187f90, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 11 / thread# 11 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee51400, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bee29721, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fcafbf1c, fcafbf18, 0, 3230d0, 187e48)
0004d1e4 ???????? (187fb0, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (187fb0, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 12 / thread# 12 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee51600, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bee165f4, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fc9fbf1c, fc9fbf18, 0, 1f0338, 187e48)
0004d1e4 ???????? (187fd0, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (187fd0, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 13 / thread# 13 --------------------
fef9fc00 _lwp_suspend (4d7080, 4dcfa8, 4, 4dcfa8, 0, 1) + 18
00045af0 ???????? (4d7088, 4d6f98, 4d6f98, 4a, 78088, 1bb820)
00042184 ???????? (4d7088, 4d6f98, 4d6f98, 4a, 4d7080, 4d8f58)
0004cb3c ???????? (4d6f50, 4d6f98, 1, a, 4d8f58, 187e48)
0004d28c ???????? (187ff0, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (187ff0, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 14 / thread# 14 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee51a00, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bed5f444, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fc7fbf1c, fc7fbf18, 0, 448850, 187e48)
0004d1e4 ???????? (188010, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (188010, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 15 / thread# 15 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee51c00, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bedd9564, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fc6fbf1c, fc6fbf18, 0, 341148, 187e48)
0004d1e4 ???????? (188030, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (188030, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 16 / thread# 16 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee51e00, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bedf40a3, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fc5fbf1c, fc5fbf18, 0, 1f0338, 187e48)
0004d1e4 ???????? (188050, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (188050, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 17 / thread# 17 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee52000, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bee31133, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fc4fbf1c, fc4fbf18, 0, 3a48a0, 187e48)
0004d1e4 ???????? (188070, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (188070, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 18 / thread# 18 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee52200, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bedca140, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fc3fbf1c, fc3fbf18, 0, 3230d0, 187e48)
0004d1e4 ???????? (188090, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (188090, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 19 / thread# 19 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee52400, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bed4c317, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fc2fbf1c, fc2fbf18, 0, 309068, 187e48)
0004d1e4 ???????? (1880b0, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (1880b0, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 20 / thread# 20 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee52600, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bee034c7, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fc1fbf1c, fc1fbf18, 0, 1f0338, 187e48)
0004d1e4 ???????? (1880d0, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (1880d0, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 21 / thread# 21 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee52800, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bed50020, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fc0fbf1c, fc0fbf18, 0, 245620, 187e48)
0004d1e4 ???????? (1880f0, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (1880f0, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 22 / thread# 22 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee52a00, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bed7dc8c, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fbffbf1c, fbffbf18, 0, 3a48a0, 187e48)
0004d1e4 ???????? (188110, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (188110, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 23 / thread# 23 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee52c00, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bedb7013, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fbefbf1c, fbefbf18, 0, 3a48a0, 187e48)
0004d1e4 ???????? (188130, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (188130, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 24 / thread# 24 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee52e00, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bee21d0f, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fbdfbf1c, fbdfbf18, 0, 4e8f98, 187e48)
0004d1e4 ???????? (188150, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (188150, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 25 / thread# 25 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee53000, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bedaf601, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fbcfbf1c, fbcfbf18, 0, 448850, 187e48)
0004d1e4 ???????? (188170, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (188170, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 26 / thread# 26 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee53200, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, beda01dd, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fbbfbf1c, fbbfbf18, 0, 3230d0, 187e48)
0004d1e4 ???????? (188190, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (188190, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 27 / thread# 27 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee53400, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bedec691, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fbafbf1c, fbafbf18, 0, 3a48a0, 187e48)
0004d1e4 ???????? (1881b0, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (1881b0, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 28 / thread# 28 --------------------
fef9c330 lseek64 (16, 7, ff2c1bb0, 0, 0, 0) + 24
fee6df28 fcntl (16, 7, ff2c1bb0, 0, fee51400, ff2c1cf4) + 74
ff2a29d0 proc_mutex_fcntl_acquire (186638, 33f180, 3a28d8, 1, 187cf0,
ff2c1b5c) + 24
ff2a2cf4 apr_proc_mutex_lock (186638, fb9fbf00, 1ee330, e4bb0, fb9fbef8,
81df8) + c
0004d01c ???????? (81b68, 10, 1881d0, 78000, 1, 77400)
ff2ad898 dummy_worker (1881d0, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 1 / thread# 1 --------------------
fef9ee28 ProcUpdate (a, ffbff873, 1, 187c60, 187c60, 328) + 5c
0004fa64 ???????? (15fc58, 4cbc4, 4d3c0, 187c38, 0, 1)
0004dab8 ???????? (1, 4c6f8, 0, 0, 76c00, 0)
0004dd58 ???????? (77400, 1, 0, 0, 11, 78000)
0004e0cc ???????? (1, 1, 11, 11, 28, 1)
0004e28c ???????? (0, 6cc, 7, 0, 78400, 78000)
0004e5c8 ???????? (837a8, b1860, 0, 856a0, 78000, 78000)
000291f0 ???????? (75c00, 856a0, 77800, 77800, 0, 0)
000282d0 ???????? (0, 0, 0, 0, 0, 0)
----------------- lwp# 3 / thread# 3 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee50400, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bede4c7f, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fd2fbf1c, fd2fbf18, 0, 3a48a0, 187e48)
0004d1e4 ???????? (187eb0, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (187eb0, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 4 / thread# 4 --------------------
fee758f8 __fork (187cd0, fee88b48, 0, 0, fee50600, fee88000) + 2c
fee73358 cond_wait (187cd0, 187ca0, 11d0, bee29721, 23a, 2f) + 14
fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8
0004f96c ???????? (187c80, fd1fbf1c, fd1fbf18, 0, 245620, 187e48)
0004d1e4 ???????? (187ed0, 1, 0, 0, 0, 0)
ff2ad898 dummy_worker (187ed0, 0, 0, 0, 0, 0) + c
fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0
----------------- lwp# 2 / thread# 2 --------------------
ff2ad88c dummy_worker(), exit value = 0x00000000
** zombie (exited, not detached, not yet joined) **


It's apache 2.2.12 running as reverse proxy on Solaris 9 box with mpm_worker
module. Looks like the threads get in some conditional wait state and the
process becomes zombie. I can see individual child processes dieing in the
log file with segmentation fault.

Any idea what might be causing these core dumps?

Thanks in advance for any suggestion/idea. Please tell me if anything else
needed for troubleshooting.

Igor

--000e0cd5391ca143af047438e2d9
Content-Type: text/html; charset=ISO-8859-1
Content-Transfer-Encoding: quoted-printable

Hi all,

I get the following core dumps from the apache:


<=
font size=3D"2">core 'core.httpd.15844.wenpweb1.500.500.1253648224'=
of 15844:   ./httpd -k start

-----------------=A0 lwp# 5 / thread# 5=A0 --------------------

=A0ff29da3c apr_table_add (6c0348, 453800, 453809, 5, fffc0000, b) + b4

=A0fec521d8 deep_table_copy (6c0348, 0, 5, 4582d4, 38, 1) + 60

=A0fec522f0 store_headers (135c78, 4528b8, 135c80, 4469c8, 135c80, fec522d4=
) + 1c

=A0fec92860 cache_save_filter (0, 0, 4742f, b8f21380, 4742f, b90155c0) + 53=
8

=A0000454b8 ???????? (44e888, 450838, 7, 0, 0, fea31494)

=A0000454b8 ???????? (45d2a8, 450838, 7, 0, 4488d0, 45083c)

=A0fe8d4d18 ap_proxy_http_process_response (ffffdfec, 4528b8, 3511c8, 45689=
8, fd0f9a20, 0) + 6c0

=A0fe8d5880 proxy_http_handler (0, 1ba088, 1b9f18, 45d4d8, 0, fd0fbab0) + 2=
90

=A0fe938b4c proxy_run_scheme_handler (4528b8, 1ba088, 1b9f18, 44ebbe, 0, 0)=
+ 64

=A0fe935cf8 proxy_handler (4528b8, 70, 4528b8, 0, 1b9fa0, c9bd8) + 41c

=A00003b994 ???????? (4528b8, 0, 1b, 1bad40, 1ba7b8, 0)

=A00003bf38 ???????? (4528b8, 5cc00, 4528b8, fd0fbcdc, fd400028, 1)

=A000048444 ???????? (4528b8, 77400, 4, 4528b8, 0, 1e8480)

=A000045b00 ???????? (446980, 446890, 446890, 42, 78088, 1bb820)

=A000042184 ???????? (446980, 446890, 446890, 42, 446978, 448850)

=A00004cb3c ???????? (446848, 446890, 1, 2, 448850, 187e48)

=A00004d28c ???????? (187ef0, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (187ef0, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 6 / thread# 6=A0 --------------------

=A0fef9dbb8 _lwp_makecontext (fcffb418, 1, 1b7740, 3e8, 0, 4ada4) + 14

=A0ff2ac204 apr_wait_for_io_or_timeout (11, fcffb418, 0, 0, fcffb674, 3208)=
+ 54

=A0ff2a7114 apr_socket_sendv (1ee378, fcffb658, 3, fcffb504, 1f9712, b96) +=
50

=A00003a424 ???????? (1ee378, fcffb650, 4, 1f49, fcffb5a8, fcffb6d0)

=A00003ada8 ???????? (0, 6fd7e8, 78348, 5d2b4, 0, 0)

=A0000454b8 ???????? (1ee9e8, 6fd7e8, 611de, fcffb818, 0, fead0c24)

=A0000454b8 ???????? (1ee9a8, 6fd7e8, 2, 0, 0, 1f0358)

=A00004b004 ???????? (6fd9e0, 6fd7e8, 0, 0, 780e8, 1ee468)

=A0000454b8 ???????? (6fd9e0, 6fd7e8, 0, 0, 0, 4ada4)

=A0000454b8 ???????? (1f31c8, 6fd7e8, 0, 0, 0, 0)

=A000030b94 ???????? (1f3198, 6fd7e8, 0, 0, 0, 30abc)

=A0000454b8 ???????? (1f3198, 6fd7e8, 1, 0, 0, 1f06b8)

=A0ff3270f8 apr_brigade_write (6fd7e8, 45648, 1f3198, fcffbaef, 1, 1ee468) =
+ 168

=A000030f88 ???????? (ffffffff, 2e, 0, 0, 6eddc8, fe8249f8)

=A0fe811f98 status_handler (20c, 0, 0, 1f2380, fe8249fc, fe8249f8) + 710 >
=A00003b994 ???????? (1f2380, 0, f, 1f4160, 1f4060, 0)

=A00003bf38 ???????? (1f2380, 5cc00, 1f2380, fcffbcdc, fd400028, 1)

=A000048444 ???????? (1f2380, 77400, 4, 1f2380, 78088, 1baf38)

=A000045b00 ???????? (1ee468, 1ee378, 1ee378, 43, 78088, 1bb820)

=A000042184 ???????? (1ee468, 1ee378, 1ee378, 43, 1ee460, 1f0338)

=A00004cb3c ???????? (1ee330, 1ee378, 1, 3, 1f0338, 187e48)

=A00004d28c ???????? (187f10, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (187f10, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 7 / thread# 7=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee50c00, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bed8d0b0, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fcefbf1c, fcefbf18, 0, 3a48a0, 187e48)

=A00004d1e4 ???????? (187f30, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (187f30, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 8 / thread# 8=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee50e00, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bed81995, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fcdfbf1c, fcdfbf18, 0, 309068, 187e48)

=A00004d1e4 ???????? (187f50, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (187f50, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 9 / thread# 9=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee51000, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bed6e868, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fccfbf1c, fccfbf18, 0, 1f0338, 187e48)

=A00004d1e4 ???????? (187f70, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (187f70, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 10 / thread# 10=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee51200, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bee29721, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fcbfbf1c, fcbfbf18, 0, 3a48a0, 187e48)

=A00004d1e4 ???????? (187f90, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (187f90, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 11 / thread# 11=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee51400, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bee29721, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fcafbf1c, fcafbf18, 0, 3230d0, 187e48)

=A00004d1e4 ???????? (187fb0, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (187fb0, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 12 / thread# 12=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee51600, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bee165f4, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fc9fbf1c, fc9fbf18, 0, 1f0338, 187e48)

=A00004d1e4 ???????? (187fd0, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (187fd0, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 13 / thread# 13=A0 --------------------

=A0fef9fc00 _lwp_suspend (4d7080, 4dcfa8, 4, 4dcfa8, 0, 1) + 18

=A000045af0 ???????? (4d7088, 4d6f98, 4d6f98, 4a, 78088, 1bb820)

=A000042184 ???????? (4d7088, 4d6f98, 4d6f98, 4a, 4d7080, 4d8f58)

=A00004cb3c ???????? (4d6f50, 4d6f98, 1, a, 4d8f58, 187e48)

=A00004d28c ???????? (187ff0, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (187ff0, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 14 / thread# 14=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee51a00, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bed5f444, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fc7fbf1c, fc7fbf18, 0, 448850, 187e48)

=A00004d1e4 ???????? (188010, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (188010, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 15 / thread# 15=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee51c00, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bedd9564, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fc6fbf1c, fc6fbf18, 0, 341148, 187e48)

=A00004d1e4 ???????? (188030, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (188030, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 16 / thread# 16=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee51e00, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bedf40a3, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fc5fbf1c, fc5fbf18, 0, 1f0338, 187e48)

=A00004d1e4 ???????? (188050, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (188050, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 17 / thread# 17=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee52000, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bee31133, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fc4fbf1c, fc4fbf18, 0, 3a48a0, 187e48)

=A00004d1e4 ???????? (188070, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (188070, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 18 / thread# 18=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee52200, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bedca140, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fc3fbf1c, fc3fbf18, 0, 3230d0, 187e48)

=A00004d1e4 ???????? (188090, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (188090, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 19 / thread# 19=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee52400, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bed4c317, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fc2fbf1c, fc2fbf18, 0, 309068, 187e48)

=A00004d1e4 ???????? (1880b0, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (1880b0, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 20 / thread# 20=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee52600, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bee034c7, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fc1fbf1c, fc1fbf18, 0, 1f0338, 187e48)

=A00004d1e4 ???????? (1880d0, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (1880d0, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 21 / thread# 21=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee52800, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bed50020, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fc0fbf1c, fc0fbf18, 0, 245620, 187e48)

=A00004d1e4 ???????? (1880f0, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (1880f0, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 22 / thread# 22=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee52a00, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bed7dc8c, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fbffbf1c, fbffbf18, 0, 3a48a0, 187e48)

=A00004d1e4 ???????? (188110, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (188110, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 23 / thread# 23=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee52c00, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bedb7013, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fbefbf1c, fbefbf18, 0, 3a48a0, 187e48)

=A00004d1e4 ???????? (188130, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (188130, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 24 / thread# 24=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee52e00, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bee21d0f, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fbdfbf1c, fbdfbf18, 0, 4e8f98, 187e48)

=A00004d1e4 ???????? (188150, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (188150, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 25 / thread# 25=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee53000, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bedaf601, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fbcfbf1c, fbcfbf18, 0, 448850, 187e48)

=A00004d1e4 ???????? (188170, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (188170, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 26 / thread# 26=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee53200, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, beda01dd, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fbbfbf1c, fbbfbf18, 0, 3230d0, 187e48)

=A00004d1e4 ???????? (188190, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (188190, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 27 / thread# 27=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee53400, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bedec691, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fbafbf1c, fbafbf18, 0, 3a48a0, 187e48)

=A00004d1e4 ???????? (1881b0, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (1881b0, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 28 / thread# 28=A0 --------------------

=A0fef9c330 lseek64=A0 (16, 7, ff2c1bb0, 0, 0, 0) + 24

=A0fee6df28 fcntl  =A0 (16, 7, ff2c1bb0, 0, fee51400, ff2c1cf4) + 74 >
=A0ff2a29d0 proc_mutex_fcntl_acquire (186638, 33f180, 3a28d8, 1, 187cf0, ff=
2c1b5c) + 24

=A0ff2a2cf4 apr_proc_mutex_lock (186638, fb9fbf00, 1ee330, e4bb0, fb9fbef8,=
81df8) + c

=A00004d01c ???????? (81b68, 10, 1881d0, 78000, 1, 77400)

=A0ff2ad898 dummy_worker (1881d0, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 1 / thread# 1=A0 --------------------

=A0fef9ee28 ProcUpdate (a, ffbff873, 1, 187c60, 187c60, 328) + 5c

=A00004fa64 ???????? (15fc58, 4cbc4, 4d3c0, 187c38, 0, 1)

=A00004dab8 ???????? (1, 4c6f8, 0, 0, 76c00, 0)

=A00004dd58 ???????? (77400, 1, 0, 0, 11, 78000)

=A00004e0cc ???????? (1, 1, 11, 11, 28, 1)

=A00004e28c ???????? (0, 6cc, 7, 0, 78400, 78000)

=A00004e5c8 ???????? (837a8, b1860, 0, 856a0, 78000, 78000)

=A0000291f0 ???????? (75c00, 856a0, 77800, 77800, 0, 0)

=A0000282d0 ???????? (0, 0, 0, 0, 0, 0)

-----------------=A0 lwp# 3 / thread# 3=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee50400, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bede4c7f, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fd2fbf1c, fd2fbf18, 0, 3a48a0, 187e48)

=A00004d1e4 ???????? (187eb0, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (187eb0, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 4 / thread# 4=A0 --------------------

=A0fee758f8 __fork   (187cd0, fee88b48, 0, 0, fee50600, fee88000) + 2c<=
br>
=A0fee73358 cond_wait (187cd0, 187ca0, 11d0, bee29721, 23a, 2f) + 14

=A0fee73394 pthread_cond_wait (187cd0, 187ca0, 0, 0, 0, 0) + 8

=A00004f96c ???????? (187c80, fd1fbf1c, fd1fbf18, 0, 245620, 187e48)

=A00004d1e4 ???????? (187ed0, 1, 0, 0, 0, 0)

=A0ff2ad898 dummy_worker (187ed0, 0, 0, 0, 0, 0) + c

=A0fee757b4 __csigsetjmp (0, 0, 0, 0, 0, 0) + b0

-----------------=A0 lwp# 2 / thread# 2=A0 --------------------

=A0ff2ad88c dummy_worker(), exit value =3D 0x00000000

      =A0 ** zombie (exited, not detached, not yet joined) ** r>

It's apache 2.2.12 running as reverse proxy on Solaris 9 box =
with mpm_worker module. Looks like the threads get in some conditional wait=
state and the process becomes zombie. I can see individual child processes=
dieing in the log file with segmentation fault.


Any idea what might be causing these core dumps?

Thanks in advan=
ce for any suggestion/idea. Please tell me if anything else needed for trou=
bleshooting.

Igor



--000e0cd5391ca143af047438e2d9--

Re: apache2 core dump

On 23 Sep 2009, at 07:32, Igor Cicimov wrote:

> Hi all,
>
> I get the following core dumps from the apache:
> [chop]
>
> It's apache 2.2.12 running as reverse proxy on Solaris 9 box with
> mpm_worker module. Looks like the threads get in some conditional
> wait state and the process becomes zombie. I can see individual
> child processes dieing in the log file with segmentation fault.

What tool produced that dump? Do you have gdb available?
(a gdb backtrace means more to me than what you show).

You mention it's a reverse proxy, and the first thread you show is
a proxy request. Are the crashes always associated with that,
so we could look there for a cause? Or is that a meaningless
question because the server does nothing else?

You might also apply Jeff's APR patch at
https://issues.apache.org/bugzilla/attachment.cgi?id=24161
and see if that helps.

--
Nick Kew

------------------------------------------------------------ ---------
The official User-To-User support forum of the Apache HTTP Server Project.
See for more info.
To unsubscribe, e-mail: users-unsubscribe@httpd.apache.org
" from the digest: users-digest-unsubscribe@httpd.apache.org
For additional commands, e-mail: users-help@httpd.apache.org

Re: apache2 core dump

--000e0cd25dd4dfd7a1047449b73f
Content-Type: text/plain; charset=ISO-8859-1

Hi Nick,

First thanks for your reply much appreciate it.

Yes, the server is reverse proxy only and from what I could see from couple
of other core dumps they all look the same. First I thought it might be the
proxy_html module causing this but now I'm not that sure and might be
something more serious.

I'll have a look in the patch and see if it is applicable to me.

Thanks,

Igor

On Wed, Sep 23, 2009 at 6:55 PM, Nick Kew wrote:

>
> On 23 Sep 2009, at 07:32, Igor Cicimov wrote:
>
> Hi all,
>>
>> I get the following core dumps from the apache:
>> [chop]
>>
>> It's apache 2.2.12 running as reverse proxy on Solaris 9 box with
>> mpm_worker module. Looks like the threads get in some conditional wait state
>> and the process becomes zombie. I can see individual child processes dieing
>> in the log file with segmentation fault.
>>
>
> What tool produced that dump? Do you have gdb available?
> (a gdb backtrace means more to me than what you show).
>
> You mention it's a reverse proxy, and the first thread you show is
> a proxy request. Are the crashes always associated with that,
> so we could look there for a cause? Or is that a meaningless
> question because the server does nothing else?
>
> You might also apply Jeff's APR patch at
> https://issues.apache.org/bugzilla/attachment.cgi?id=24161
> and see if that helps.
>
> --
> Nick Kew
>
> ------------------------------------------------------------ ---------
> The official User-To-User support forum of the Apache HTTP Server Project.
> See for more info.
> To unsubscribe, e-mail: users-unsubscribe@httpd.apache.org
> " from the digest: users-digest-unsubscribe@httpd.apache.org
> For additional commands, e-mail: users-help@httpd.apache.org
>
>

--000e0cd25dd4dfd7a1047449b73f
Content-Type: text/html; charset=ISO-8859-1
Content-Transfer-Encoding: quoted-printable

Hi Nick,

First thanks for your reply much appreciate it.

Yes,=
the server is reverse proxy only and from what I could see from couple of =
other core dumps they all look the same. First I thought it might be the pr=
oxy_html module causing this but now I'm not that sure and might be som=
ething more serious.


I'll have a look in the patch and see if it is applicable to me. >
Thanks,

Igor

Re: apache2 core dump

--000e0cd29f8c11681b047449c01d
Content-Type: text/plain; charset=ISO-8859-1

Ah forgot about the tool ... used pstack to debug the core dumps since it is
a production server and don't have gdb available.

Cheers,

Igor

On Thu, Sep 24, 2009 at 12:37 PM, Igor Cicimov wrote:

> Hi Nick,
>
> First thanks for your reply much appreciate it.
>
> Yes, the server is reverse proxy only and from what I could see from couple
> of other core dumps they all look the same. First I thought it might be the
> proxy_html module causing this but now I'm not that sure and might be
> something more serious.
>
> I'll have a look in the patch and see if it is applicable to me.
>
> Thanks,
>
> Igor
>
>
> On Wed, Sep 23, 2009 at 6:55 PM, Nick Kew wrote:
>
>>
>> On 23 Sep 2009, at 07:32, Igor Cicimov wrote:
>>
>> Hi all,
>>>
>>> I get the following core dumps from the apache:
>>> [chop]
>>>
>>> It's apache 2.2.12 running as reverse proxy on Solaris 9 box with
>>> mpm_worker module. Looks like the threads get in some conditional wait state
>>> and the process becomes zombie. I can see individual child processes dieing
>>> in the log file with segmentation fault.
>>>
>>
>> What tool produced that dump? Do you have gdb available?
>> (a gdb backtrace means more to me than what you show).
>>
>> You mention it's a reverse proxy, and the first thread you show is
>> a proxy request. Are the crashes always associated with that,
>> so we could look there for a cause? Or is that a meaningless
>> question because the server does nothing else?
>>
>> You might also apply Jeff's APR patch at
>> https://issues.apache.org/bugzilla/attachment.cgi?id=24161
>> and see if that helps.
>>
>> --
>> Nick Kew
>>
>> ------------------------------------------------------------ ---------
>> The official User-To-User support forum of the Apache HTTP Server Project.
>> See for more info.
>> To unsubscribe, e-mail: users-unsubscribe@httpd.apache.org
>> " from the digest: users-digest-unsubscribe@httpd.apache.org
>> For additional commands, e-mail: users-help@httpd.apache.org
>>
>>
>

--000e0cd29f8c11681b047449c01d
Content-Type: text/html; charset=ISO-8859-1
Content-Transfer-Encoding: quoted-printable

Ah forgot about the tool ... used pstack to debug the core dumps since it i=
s a production server and don't have gdb available.

Cheers,
<=
br>Igor

Re: apache2 core dump

Thought to post some more debugging regarding this problem I have.
This time I had help from one of our Unix engineers with the debugging
and you can see he used mdb in this case:

bash-2.05$ mdb core.httpd.28499.wenpweb1.500.500.1253659663

mdb: warning: failed to infer pathname to executable; symbol table
will not be available

Loading modules: [ libc.so.1 libthread.so.1 ld.so.1 ]

> $C (display call stack)

fbef95d8 libapr-1.so.0`apr_palloc+4(44617465, 7, 70740020, 7efefeff,
81010100, ff00)

fbef9648 libapr-1.so.0`apr_pstrdup+0x20(44617465, 27a758, 0, 13f75a4, 0, 27a75a)

fbef96b8 libapr-1.so.0`apr_table_add+0xa8(13f75a0, 27a758, 27a760, 5,
fffc0000, d)

fbef9728 mod_mem_cache.so`deep_table_copy+0x60(13f75a0, 0, 5, 2ba944, 38, 1)

fbef9798 mod_mem_cache.so`store_headers+0x1c(ecba0, 2797f8, ecba8,
26b900, ecba8, fec522d4)

(snip)

> apr_palloc::dis (disassemble apr_palloc)

libapr-1.so.0`apr_palloc: save %sp, -0x70, %sp

libapr-1.so.0`apr_palloc+4: ld [%i0 + 0x2c], %l0

libapr-1.so.0`apr_palloc+8: mov %i0, %l1

(snip)

> $r (display registers)

(snip)

%o0 = 0x00008ca0 %i0 = 0x44617465

(snip)





%i0 (the first parameter being passed to a function) is an invalid
pointer. Looking at it, it actually appears to be part of a text
string:

0x44 = D

0x61 = a

0x74 = t

0x65 = e



Putting all this together, apr_table_add is dereferencing a string and
passing the contents of it to apr_pstrdup, instead of passing a
pointer to the memory pool (which is what apr_pstrdup expects)



Most likely something in apr_table_add is getting over-written.



Other cores show similar behaviour:



hmurn@wenpweb1# mdb core.httpd.28752.wenpweb1.500.500.1254735017

> $C

fcff95d8 libapr-1.so.0`apr_palloc+4(436f6e74 ('Cont'), 7, 70740020,
7efefeff, 81010100, ff00)


bash-2.05# mdb core.httpd.12737.wenpweb1.500.500.1254699635

> $C

fccf96b8 libapr-1.so.0`apr_table_add+0xb4(56f5c10, 527d0f8, 5281740, 5, 1, 6)

> apr_table_add+0xb4::dis

libapr-1.so.0`apr_table_add+0xb4: st %o0, [%l0]

> $r

%g0 = 0x00000000 %l0 = 0x68747470 ('http')


So it really looks like a bug in the apache apr so the patch that Nick
suggested makes more sense for me now.

Nik, do you know if this is the same issue that the patch is suppose
to fix? Can you please point me to the patch release documentation so
I can read some more about it?

Thanks for your help.

Igor

On 9/24/09, Igor Cicimov wrote:
> Ah forgot about the tool ... used pstack to debug the core dumps since it
> is
> a production server and don't have gdb available.
>
> Cheers,
>
> Igor
>
> On Thu, Sep 24, 2009 at 12:37 PM, Igor Cicimov wrote:
>
>> Hi Nick,
>>
>> First thanks for your reply much appreciate it.
>>
>> Yes, the server is reverse proxy only and from what I could see from
>> couple
>> of other core dumps they all look the same. First I thought it might be
>> the
>> proxy_html module causing this but now I'm not that sure and might be
>> something more serious.
>>
>> I'll have a look in the patch and see if it is applicable to me.
>>
>> Thanks,
>>
>> Igor
>>
>>
>> On Wed, Sep 23, 2009 at 6:55 PM, Nick Kew wrote:
>>
>>>
>>> On 23 Sep 2009, at 07:32, Igor Cicimov wrote:
>>>
>>> Hi all,
>>>>
>>>> I get the following core dumps from the apache:
>>>> [chop]
>>>>
>>>> It's apache 2.2.12 running as reverse proxy on Solaris 9 box with
>>>> mpm_worker module. Looks like the threads get in some conditional wait
>>>> state
>>>> and the process becomes zombie. I can see individual child processes
>>>> dieing
>>>> in the log file with segmentation fault.
>>>>
>>>
>>> What tool produced that dump? Do you have gdb available?
>>> (a gdb backtrace means more to me than what you show).
>>>
>>> You mention it's a reverse proxy, and the first thread you show is
>>> a proxy request. Are the crashes always associated with that,
>>> so we could look there for a cause? Or is that a meaningless
>>> question because the server does nothing else?
>>>
>>> You might also apply Jeff's APR patch at
>>> https://issues.apache.org/bugzilla/attachment.cgi?id=24161
>>> and see if that helps.
>>>
>>> --
>>> Nick Kew
>>>
>>> ------------------------------------------------------------ ---------
>>> The official User-To-User support forum of the Apache HTTP Server
>>> Project.
>>> See for more info.
>>> To unsubscribe, e-mail: users-unsubscribe@httpd.apache.org
>>> " from the digest: users-digest-unsubscribe@httpd.apache.org
>>> For additional commands, e-mail: users-help@httpd.apache.org
>>>
>>>
>>
>

------------------------------------------------------------ ---------
The official User-To-User support forum of the Apache HTTP Server Project.
See for more info.
To unsubscribe, e-mail: users-unsubscribe@httpd.apache.org
" from the digest: users-digest-unsubscribe@httpd.apache.org
For additional commands, e-mail: users-help@httpd.apache.org

Re: apache2 core dump

That looks like Apache PR 47672, which has been fixed. Can you try
Apache 2.2.14?

Dan Poirier

------------------------------------------------------------ ---------
The official User-To-User support forum of the Apache HTTP Server Project.
See for more info.
To unsubscribe, e-mail: users-unsubscribe@httpd.apache.org
" from the digest: users-digest-unsubscribe@httpd.apache.org
For additional commands, e-mail: users-help@httpd.apache.org

Re: Re: apache2 core dump

Hi Dan, thanks for your reply. I'm not sure about 2.2.14 since this is
production server. I'll have to build it first in model and test it.
Strangely I didn't face the same problem on the model servers with the
same Solaris 9 OS but that might have been because of lack of traffic.

Is there any other way to fix this, a patch that I can apply maybe?

Thanks,
Igor

On 10/8/09, Dan Poirier wrote:
> That looks like Apache PR 47672, which has been fixed. Can you try
> Apache 2.2.14?
>
> Dan Poirier
>
> ------------------------------------------------------------ ---------
> The official User-To-User support forum of the Apache HTTP Server Project.
> See for more info.
> To unsubscribe, e-mail: users-unsubscribe@httpd.apache.org
> " from the digest: users-digest-unsubscribe@httpd.apache.org
> For additional commands, e-mail: users-help@httpd.apache.org
>
>

------------------------------------------------------------ ---------
The official User-To-User support forum of the Apache HTTP Server Project.
See for more info.
To unsubscribe, e-mail: users-unsubscribe@httpd.apache.org
" from the digest: users-digest-unsubscribe@httpd.apache.org
For additional commands, e-mail: users-help@httpd.apache.org

Re: Re: apache2 core dump

--000e0cd2c1e045248104757ac8f2
Content-Type: text/plain; charset=ISO-8859-1

Hi,

I tried to recompile and install the patched mod_mem_cache.c file from the
bug report above but I got some errors. The command I used to create and
install the loadable module:

# /usr/local/apache2/bin/apxs -c -i mod_mem_cache.c cache_storage.c
cache_util.c

This created new mod_mem_cache.so file in the modules folder but when I
tried to start the server I got the following error:


httpd: Syntax error on line 70 of /usr/local/apache2/conf/httpd.conf: Cannot
load /usr/local/apache2/modules/mod_mem_cache.so into server: ld.so.1:
../bin/httpd: fatal: relocation error: file
/usr/local/apache2/modules/mod_mem_cache.so: symbol cache_find: referenced
symbol not found

I don't know if the way I did it is the correct one and if the error I get
is maybe related with some problem installing the module on Solaris 9 (I
have the same problem with the latest mod_security.2.5.10)?

Thanks,

Igor

On Thu, Oct 8, 2009 at 11:57 PM, Igor Cicimov wrote:

> Hi Dan, thanks for your reply. I'm not sure about 2.2.14 since this is
> production server. I'll have to build it first in model and test it.
> Strangely I didn't face the same problem on the model servers with the
> same Solaris 9 OS but that might have been because of lack of traffic.
>
> Is there any other way to fix this, a patch that I can apply maybe?
>
> Thanks,
> Igor
>
> On 10/8/09, Dan Poirier wrote:
> > That looks like Apache PR 47672, which has been fixed. Can you try
> > Apache 2.2.14?
> >
> > Dan Poirier
> >
> > ------------------------------------------------------------ ---------
> > The official User-To-User support forum of the Apache HTTP Server
> Project.
> > See for more info.
> > To unsubscribe, e-mail: users-unsubscribe@httpd.apache.org
> > " from the digest: users-digest-unsubscribe@httpd.apache.org
> > For additional commands, e-mail: users-help@httpd.apache.org
> >
> >
>

--000e0cd2c1e045248104757ac8f2
Content-Type: text/html; charset=ISO-8859-1
Content-Transfer-Encoding: quoted-printable

Hi,

I tried to recompile and install the patched mod_mem_cache.c fil=
e from the bug report above but I got some errors. The command I used to cr=
eate and install the loadable module:

# /usr/local/apache2/bin/apxs =
-c -i mod_mem_cache.c cache_storage.c cache_util.c


This created new mod_mem_cache.so file in the modules folder but when I=
tried to start the server I got the following error:


httpd: Syn=
tax error on line 70 of /usr/local/apache2/conf/httpd.conf: Cannot load /us=
r/local/apache2/modules/mod_mem_cache.so into server: ld.so.1: ./bin/httpd:=
fatal: relocation error: file /usr/local/apache2/modules/mod_mem_cache.so:=
symbol cache_find: referenced symbol not found


I don't know if the way I did it is the correct one and if the erro=
r I get is maybe related with some problem installing the module on Solaris=
9 (I have the same problem with the latest mod_security.2.5.10)?


Thanks,

Igor